repo: Store transaction file, use it to optimize for pull resumes

If pull is interrupted, we may have downloaded an arbitrary subset of
the requested objects.  Previously, we handled this by scanning for
all objects each time.

However, there's an easy optimization - this patch creates a lock file
in the repo.  If we don't see that file when starting a pull, we know
we don't need to stat() every file; presence of a dirtree object for
example implies the existence of everything it references.

diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 0975e72a6d6921ae5b936f353d85454039f13cba..016d3616891d2f68529284dd5577e2f50d414ab7 100644 (file)
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -54,6 +54,8 @@ struct OstreeRepo {
   GFile *remote_cache_dir;
   GFile *config_file;
 
+  GFile *transaction_lock_path;
+
   GMutex cache_lock;
   GPtrArray *cached_meta_indexes;
   GPtrArray *cached_content_indexes;
@@ -106,6 +108,9 @@ ostree_repo_finalize (GObject *object)
   g_clear_object (&self->uncompressed_objects_dir);
   g_clear_object (&self->remote_cache_dir);
   g_clear_object (&self->config_file);
+
+  g_clear_object (&self->transaction_lock_path);
+
   if (self->loose_object_devino_hash)
     g_hash_table_destroy (self->loose_object_devino_hash);
   if (self->updated_uncompressed_dirs)
@@ -1357,14 +1362,34 @@ devino_cache_lookup (OstreeRepo           *self,
 gboolean
 ostree_repo_prepare_transaction (OstreeRepo     *self,
                                  gboolean        enable_commit_hardlink_scan,
+                                 gboolean       *out_transaction_resume,
                                  GCancellable   *cancellable,
                                  GError        **error)
 {
   gboolean ret = FALSE;
+  gboolean ret_transaction_resume = FALSE;
+  gs_free char *transaction_str = NULL;
 
   g_return_val_if_fail (self->in_transaction == FALSE, FALSE);
 
+  if (self->transaction_lock_path == NULL)
+    self->transaction_lock_path = g_file_resolve_relative_path (self->repodir, "transaction");
+
+  if (g_file_query_file_type (self->transaction_lock_path, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL) == G_FILE_TYPE_SYMBOLIC_LINK)
+    ret_transaction_resume = TRUE;
+  else
+    ret_transaction_resume = FALSE;
+
   self->in_transaction = TRUE;
+  if (ret_transaction_resume)
+    {
+      if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error))
+        goto out;
+    }
+  transaction_str = g_strdup_printf ("pid=%llu", (unsigned long long) getpid ());
+  if (!g_file_make_symbolic_link (self->transaction_lock_path, transaction_str,
+                                  cancellable, error))
+    goto out;
 
   if (enable_commit_hardlink_scan)
     {
@@ -1376,6 +1401,8 @@ ostree_repo_prepare_transaction (OstreeRepo     *self,
     }
 
   ret = TRUE;
+  if (out_transaction_resume)
+    *out_transaction_resume = ret_transaction_resume;
  out:
   return ret;
 }
@@ -1389,12 +1416,15 @@ ostree_repo_commit_transaction (OstreeRepo     *self,
 
   g_return_val_if_fail (self->in_transaction == TRUE, FALSE);
 
-  ret = TRUE;
-  /* out: */
-  self->in_transaction = FALSE;
+  if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error))
+    goto out;
+
   if (self->loose_object_devino_hash)
     g_hash_table_remove_all (self->loose_object_devino_hash);
 
+  self->in_transaction = FALSE;
+  ret = TRUE;
+ out:
   return ret;
 }
 

diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 69b4dff7403fc4c806d24d96987076b1de7e7233..6417b91724ddf3adf3c1031774e5e231c8331d4a 100644 (file)
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -78,6 +78,7 @@ GFile *       ostree_repo_get_file_object_path (OstreeRepo   *self,
 
 gboolean      ostree_repo_prepare_transaction (OstreeRepo     *self,
                                                gboolean        enable_commit_hardlink_scan,
+                                               gboolean       *out_transaction_resume,
                                                GCancellable   *cancellable,
                                                GError        **error);
 

diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index 16af072055b945542a5b242f419c526a7ff7d52d..eb7cc403281423c7bddd118dacf34feabd258adc 100644 (file)
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -108,6 +108,7 @@ typedef struct {
   GMainLoop    *loop;
   GCancellable *cancellable;
 
+  gboolean      transaction_resuming;
   volatile gint n_scanned_metadata;
   guint         outstanding_uri_requests;
 
@@ -822,7 +823,7 @@ scan_one_metadata_object (OtPullData         *pull_data,
   if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
                                cancellable, error))
     goto out;
-  
+
   if (!is_stored && !is_requested)
     {
       char *duped_checksum = g_strdup (tmp_checksum);
@@ -834,23 +835,26 @@ scan_one_metadata_object (OtPullData         *pull_data,
     }
   else if (is_stored)
     {
-      switch (objtype)
+      if (pull_data->transaction_resuming || is_requested)
         {
-        case OSTREE_OBJECT_TYPE_COMMIT:
-          if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth,
-                                   pull_data->cancellable, error))
-            goto out;
-          break;
-        case OSTREE_OBJECT_TYPE_DIR_META:
-          break;
-        case OSTREE_OBJECT_TYPE_DIR_TREE:
-          if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth,
-                                    pull_data->cancellable, error))
-            goto out;
-          break;
-        case OSTREE_OBJECT_TYPE_FILE:
-          g_assert_not_reached ();
-          break;
+          switch (objtype)
+            {
+            case OSTREE_OBJECT_TYPE_COMMIT:
+              if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth,
+                                       pull_data->cancellable, error))
+                goto out;
+              break;
+            case OSTREE_OBJECT_TYPE_DIR_META:
+              break;
+            case OSTREE_OBJECT_TYPE_DIR_TREE:
+              if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth,
+                                        pull_data->cancellable, error))
+                goto out;
+              break;
+            case OSTREE_OBJECT_TYPE_FILE:
+              g_assert_not_reached ();
+              break;
+            }
         }
       g_hash_table_insert (pull_data->scanned_metadata, g_variant_ref (object), object);
       g_atomic_int_inc (&pull_data->n_scanned_metadata);
@@ -1347,7 +1351,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
         }
     }
 
-  if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, NULL, error))
+  if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, &pull_data->transaction_resuming,
+                                        cancellable, error))
     goto out;
 
   pull_data->metadata_objects_to_fetch = ot_waitable_queue_new ();

diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c
index f01815b75f331f7c041cdb5593632cc98f9056a2..b79a3a35f7b1bb41a60469dc9fd30a6e9a764e19 100644 (file)
--- a/src/ostree/ot-builtin-commit.c
+++ b/src/ostree/ot-builtin-commit.c
@@ -351,7 +351,7 @@ ostree_builtin_commit (int argc, char **argv, GFile *repo_path, GError **error)
         goto out;
     }
 
-  if (!ostree_repo_prepare_transaction (repo, TRUE, cancellable, error))
+  if (!ostree_repo_prepare_transaction (repo, TRUE, NULL, cancellable, error))
     goto out;
 
   in_transaction = TRUE;

diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index edb19b998ba31aa1362e94748280111cd84fb4aa..471306fb71d4bfc07299c0867841673f2f456784 100644 (file)
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -170,6 +170,7 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
   int i;
   GHashTableIter hash_iter;
   gpointer key, value;
+  gboolean transaction_resuming = FALSE;
   gs_unref_hashtable GHashTable *objects = NULL;
   gs_unref_object GFile *src_f = NULL;
   gs_unref_object GFile *src_repo_dir = NULL;
@@ -247,6 +248,10 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
         }
     }
 
+  if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, &transaction_resuming,
+                                        cancellable, error))
+    goto out;
+
   g_print ("Enumerating objects...\n");
 
   source_objects = ostree_traverse_new_reachable ();
@@ -275,9 +280,6 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
         }
     }
 
-  if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, cancellable, error))
-    goto out;
-
   data->n_objects_to_check = g_hash_table_size (source_objects);
   g_hash_table_iter_init (&hash_iter, source_objects);
   while (g_hash_table_iter_next (&hash_iter, &key, &value))